#
# Copyright 2009 Red Hat, Inc. and/or its affiliates.
#
# Licensed to you under the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.  See the files README and
# LICENSE_GPL_v2 which accompany this distribution.
#

import os
from glob import glob
import logging
import time
import threading

import constants
import storage_mailbox
import blockSD
import sd
import misc
from config import config
from sdf import StorageDomainFactory as SDF
import storage_exception as se

UUID_PATTERN = '*-*-*-*-*'
BLANK_POOL_UUID = '00000000-0000-0000-0000-000000000000'

POOL_MASTER_DOMAIN = 'mastersd'

spUninit = "uninitialized"
spIniting = "initializing"
spInit = "initialized"
spConnected = "connected"
spDisconnected = "disconnected"
spConnecting = "Connecting"
spDisconnecting = "Disconnecting"
spError = "error"

MAX_POOL_DESCRIPTION_SIZE = 50

# Calculate how many domains can be in the pool before overflowing the Metadata
MAX_DOMAINS = sd.SD_METADATA_SIZE - blockSD.METADATA_BASE_SIZE
MAX_DOMAINS -= MAX_POOL_DESCRIPTION_SIZE + sd.MAX_DOMAIN_DESCRIPTION_SIZE
MAX_DOMAINS -= blockSD.PVS_METADATA_SIZE
MAX_DOMAINS /= 48

class StatsThread(threading.Thread):
    def __init__(self, func, dom):
        """
        StatsThread gets two arguments on instatiation:
        func - function to call
        dom - argument to pass to func()
        """
        threading.Thread.__init__(self)
        self._statscache = dict(result=
            dict(code=200, lastCheck=0.0, delay='0', valid=True))
        self._statsdelay = config.getint('irs', 'sd_health_check_delay')
        self._statsletrun = True
        self._statsfunc = func
        self._domain = dom


    def run(self):
        while self._statsletrun:
            try:
                start = time.time()
                stats, code = self._statsfunc(self._domain)
            except se.StorageException, e:
                misc.logException(e)
                code = e.code
            except Exception, e:
                misc.logException(e)
                code = 200

            finish = time.time()

            stats['finish'] = finish
            stats['result'] = dict(code=code, lastCheck=finish,
                delay=str(finish - start), valid=(code == 0))
            self._statscache.update(stats)

            count = 0
            while self._statsletrun and count < self._statsdelay:
                count += 1
                time.sleep(1)

        self._statsfunc = None


    def stop(self):
        self._statsletrun = False


    def getStatsResults(self):
        return self._statscache.copy()


class StoragePool:
    '''
    StoragePool object should be relatively cheap to construct. It should defer
    any heavy lifting activities until the time it is really needed.
    '''
    log = logging.getLogger('irs')
    lockUtilPath = config.get('irs', 'lock_util_path')
    lockCmd = config.get('irs', 'lock_cmd')
    freeLockCmd = config.get('irs', 'free_lock_cmd')
    lockCmd = config.get('irs', 'lock_cmd')
    domListsLock = threading.Lock()
    domLists = {}

    def __init__(self, spUUID):
        self.storage_repository = config.get('irs', 'repository')
        self._poolsTmpDir = config.get('irs', 'pools_data_dir')
        self.lvExtendPolicy = config.get('irs', 'vol_extend_policy')
        self.spUUID = str(spUUID)
        self.poolPath = os.path.join(self.storage_repository, self.spUUID)
        self.id = None
        self.scsiKey = None
        self._poolFile = os.path.join(self._poolsTmpDir, self.spUUID)
        self.hsmMailer = None
        self.spmMailer = None
        self.state = spUninit
        self.domains = {}
        self.masterDomain = None
        self.repostats = {}


    def __createMailboxMonitor(self):
        # Currently mailbox is not needed for non block device sd's
        if self.hsmMailer:
            return

        sanPool = self.getMasterDomain().getStorageType() in [ sd.ISCSI_DOMAIN, sd.FCP_DOMAIN ]
        if sanPool and self.lvExtendPolicy == "ON":
            self.hsmMailer = storage_mailbox.HSM_Mailbox(self.id, self.spUUID)


    def __cleanupDomains(self, domlist, msdUUID, masterVersion):
        """
        Clean up domains after failed Storage Pool creation
        domlist - comma separated list of sdUUIDs
        """
        # Go through all the domains and detach them from the pool
        # Since something went wrong (otherwise why would we be cleaning
        # the mess up?) do not expect all the domains to exist
        domains = [SDF.produce(d) for d in domlist]
        for d in domains:
            try:
                self.detachSD(d, msdUUID, masterVersion)
            except Exception, e:
                misc.logException(e)
        self.refresh()


    def _getLeaseParams(self, leaseParams):
        if leaseParams:
            lockRenewalIntervalSec = str(leaseParams[sd.LOCK_RENEWAL_INTERVAL_SEC])
            leaseTimeMs = str(leaseParams[sd.LEASE_TIME_SEC])
            leaseFailRetry = str(leaseParams[sd.LEASE_RETRIES])
            ioOpTimeoutMs = str(leaseParams[sd.IO_OP_TIMEOUT_SEC])
        else:
            lockRenewalIntervalSec = str(self.getMetaParam(sd.LOCK_RENEWAL_INTERVAL_SEC, cache=False))
            leaseTimeMs = str(self.getMetaParam(sd.LEASE_TIME_SEC, cache=False))
            leaseFailRetry = str(self.getMetaParam(sd.LEASE_RETRIES, cache=False))
            ioOpTimeoutMs = str(self.getMetaParam(sd.IO_OP_TIMEOUT_SEC, cache=False))
        leaseTimeMs = str(int(leaseTimeMs) * 1000)
        ioOpTimeoutMs = str(int(ioOpTimeoutMs) * 1000)
        return [lockRenewalIntervalSec, leaseTimeMs, ioOpTimeoutMs, leaseFailRetry]


    def acquireDomainLock(self, sdUUID, leaseParams=None):
        self.log.debug("Acquiring domain lock for domain %s" % sdUUID)
        grp = "lockGroup_" + self.spUUID
        self.domListsLock.acquire()
        try:
            if sdUUID in self.domLists:
                raise se.DomainAlreadyLocked(sdUUID)
            self.domLists[sdUUID] = grp
        finally:
            self.domListsLock.release()

        try:
            # Get lock parameters
            (lockRenewalIntervalSec, leaseTimeMs, ioOpTimeoutMs, leaseFailRetry) = self._getLeaseParams(leaseParams)
            domMD = SDF.produce(sdUUID).getMDPath()
            lease = os.path.join(domMD, sd.LEASES)
            lockUtil = os.path.join(self.lockUtilPath, self.lockCmd)
            # the following is run in a new process group (with setsid) because it is sometimes killed and we need to
            # avoid killing the parent (i.e. the vdsm) in these cases
            self.log.debug("sdUUID=%s, id=%s, lease=%s, lockRenewalIntervalSec=%s, leaseTimeMs=%s, ioOpTimeoutMs=%s, leaseFailRetry=%s" % (sdUUID, self.id, lease, lockRenewalIntervalSec, leaseTimeMs, ioOpTimeoutMs, leaseFailRetry))
            acquireLockCommand = ' '.join([lockUtil, "start", sdUUID, self.id,
                                            lockRenewalIntervalSec, lease, leaseTimeMs,
                                            ioOpTimeoutMs, leaseFailRetry])
            cmd = [constants.EXT_SETSID, constants.EXT_IONICE, '-c1', '-n0',
                constants.EXT_SU, misc.IOUSER, '-s', constants.EXT_SH, '-c',
                acquireLockCommand]
            (rc, out, err) = misc.execCmd(cmd, cwd=self.lockUtilPath)
            if rc != 0:
                raise se.AcquireLockFailure(sdUUID, rc, out, err)
            self.log.debug("Domain lock acquired successfully")
        except Exception, e:
            try:
                del self.domLists[sdUUID]
            except:
                pass
            raise e


    def releaseDomainLock(self, sdUUID):
        self.log.debug("Releasing domain lock")
        try:
            try:
                freeLockUtil = os.path.join(self.lockUtilPath, self.freeLockCmd)
                releaseLockCommand = [ freeLockUtil, sdUUID ]
                self.log.info("Releasing lock: %s for domain: %s" % (releaseLockCommand, sdUUID))
                (rc, out, err) = misc.execCmd(releaseLockCommand, sudo=False, cwd=self.lockUtilPath)
                if rc != 0:
                    self.log.error("Could not release domain lock rc=%s out=%s, err=%s" % (str(rc), out, err))
                self.log.debug("Domain lock released successfully")
            except Exception, e:
                misc.logException(e)
                self.log.error("release lock failed")
        finally:
            try:
                del self.domLists[sdUUID]
            except:
                pass


    def releaseGroupLock(self):
        grp = "lockGroup_" + self.spUUID
        for dom in [d for d in self.domLists if self.domLists[d] == grp]:
            try:
                self.releaseDomainLock(dom)
            except Exception, e:
                misc.logException(e)


    def isConnected(self, connecting = False):
        if connecting and self.state == spConnecting:
            return True
        return self.state == spConnected


    def validateAttachedDomain(self, sdUUID):
        domList = self.getDomains()
        if sdUUID not in domList:
            raise se.StorageDomainNotInPool(self.spUUID, sdUUID)
        # Avoid handle domains if not owned by pool
        dom = SDF.produce(sdUUID)
        pools = self.getDomainPools(dom=dom)
        if self.spUUID not in pools:
            raise se.StorageDomainNotInPool(self.spUUID, sdUUID)


    def validateUnattachedDomain(self, sdUUID):
        dom = SDF.produce(sdUUID)
        spUUID = dom.getMetaParam(sd.POOL, False)
        if spUUID:
            raise se.StorageDomainAlreadyAttached(spUUID, sdUUID)


    def validatePoolMVerEqual(self, masterVersion):
        """
        Make sure the masterVersion matches that of the pool.

        :param masterVersion: the master version you want to validate
        :type masterVersion: int

        :raises: :exc:`storage_exception.StoragePoolWrongMasterVersion`
            exception if masterVersion doesn't follow the rules

        """
        d = self.getMasterDomain()
        mver = d.getMasterVer()
        if not int(masterVersion) == mver:
            raise se.StoragePoolWrongMaster(self.spUUID, d.sdUUID)


    def validatePoolMVerHigher(self, masterVersion):
        """
        Make sure the masterVersion higher than that of the pool.

        :param masterVersion: the master version you want to validate
        :type masterVersion: int

        :raises: :exc:`storage_exception.StoragePoolWrongMasterVersion`
            exception if masterVersion doesn't follow the rules

        """
        d = self.getMasterDomain()
        mver = d.getMasterVer()
        if not int(masterVersion) > mver:
            raise se.StoragePoolWrongMaster(self.spUUID, d.sdUUID)


    def create(self, poolName, msdUUID, domList, masterVersion, safeLease):
        """
        Create new storage pool with single/multiple image data domain.
        The command will create new storage pool meta-data attach each
        storage domain to that storage pool.
        At least one data (images) domain must be provided
         'poolName' - storage pool name
         'msdUUID' - master domain of this pool (one of domList)
         'domList' - list of domains (i.e sdUUID,sdUUID,...,sdUUID)
        """
        self.log.info("(StoragePool.create) spUUID=%s poolName=%s master_sd=%s "
                      "domList=%s masterVersion=%s %s",
                      self.spUUID, poolName, msdUUID,
                      domList, masterVersion, str(safeLease))

        if msdUUID not in domList:
            raise se.InvalidParameterException("masterDomain", msdUUID)

        if self.state == spIniting:
            raise se.OperationInProgress("pool connect %s" % self.spUUID)

        if self.state != spUninit:
            raise se.StoragePoolAlreadyExists(poolName)

        self.state = spIniting

        try:
            SDF.refresh()
        except Exception, e:
            # Don't fail pool creation because of an irrelevant domain
            self.log.error("(StoragePool.create) Fail to refresh pool (%s)" % self.spUUID)
            misc.logException(e)

        # Check the domains before pool creation
        for dom in domList:
            try:
                d = SDF.produce(dom).validate()
            except se.StorageException, e:
                misc.logException(e)
                raise se.StorageDomainAccessError(dom)

            domain = SDF.produce(dom)
            if not domain.isISO():
                self.validateUnattachedDomain(sdUUID=dom)

        if not os.path.exists(self.poolPath):
            os.mkdir(self.poolPath)

        try:
            # Seeing as we are just creating the pool then the host doesn't
            # have an assigned Id for this pool.  When locking the domain we must use an Id
            self.id = "1"
            # Master domain is unattached and all changes to unattached domains
            # must be performed under storage lock
            self.acquireDomainLock(msdUUID, safeLease)
        except:
            self.id = None
            raise
        try:
            try:
                # Mark 'master' domain
                # We should do it before actually attaching this domain to the pool.
                # During 'master' marking we create pool metadata and each attached
                # domain should register there
                self.createMaster(poolName, msdUUID, masterVersion, safeLease)
                self.__rebuild(msdUUID=msdUUID, masterVersion=masterVersion)
                # Attach storage domains to the storage pool
                # Since we are creating the pool then attach is done from the hsm and not the spm
                # therefore we must manually take the master domain lock
                # TBD: create will receive only master domain and further attaches should be done
                # under SPM

                # Master domain was already attached (in createMaster),
                # no need to reattach
                for sdUUID in [d for d in domList if d != msdUUID]:
                        self.attachSD(sdUUID)
            except Exception, e:
                misc.logException(e)
                try:
                    self.log.error("create cleanup - due to %s" % str(e))
                    misc.cleanupdir(self.poolPath)
                    self.__cleanupDomains(domList, msdUUID, masterVersion)
                finally:
                    self.state = spUninit
                    raise e
        finally:
            self.releaseDomainLock(msdUUID)
            self.id = None

        self.state = spInit
        return True


    def connect(self, id, scsiKey, msdUUID, masterVersion):
        """
        Connect a Host to a specific storage pool.
        """
        self.log.info("(SP.connect) Connect host #%s to the storage pool %s with master domain: %s (ver = %s)" %
            (id, self.spUUID, msdUUID, masterVersion))
        if self.state == spConnected:
            return True
        if self.state == spConnecting:
            raise se.OperationInProgress("pool connect %s" % self.spUUID)
        self.state = spConnecting
        try:
            # Make sure SDCache doesn't have stale data (it can be in case of FC)
            SDF.refresh()

            if msdUUID and msdUUID != sd.BLANK_UUID:
                # We should find the master domain with the proper
                # version. If we don't we should raise
                # the special exception StoragePoolMasterNotFound
                # and VDC will reconstruct master on another domain
                # if available
                mDom = self.getMasterDomain(msdUUID=msdUUID, masterVersion=masterVersion)

            if not os.path.exists(self._poolsTmpDir):
                msg = ("StoragePoolConnectionError for hostId: %s, on poolId: %s," +
                       " Pools temp data dir: %s does not exist" %
                        (id, self.spUUID, self._poolsTmpDir))
                self.log.error(msg)
                msg = "Pools temp data dir: %s does not exist" % (self._poolsTmpDir)
                raise se.StoragePoolConnectionError(msg)
            if os.path.exists(self._poolFile):
                os.unlink(self._poolFile)
            pers = ["id=%s\n" % id]
            pers.append("scsiKey=%s\n" % scsiKey)
            pers.append("sdUUID=%s\n" % msdUUID)
            pers.append("version=%s\n" %masterVersion)
            open(self._poolFile, "w").writelines(pers)
            self.id = str(id)
            self.scsiKey = scsiKey

            # Rebuild whole Pool
            self.__rebuild()
            self.__createMailboxMonitor()
        except:
            self.state = spDisconnected
            raise

        self.state = spConnected
        return True


    def disconnectDomains(self):
        try:
            domDict = self.getDomains(activeOnly=True)
        except Exception, e:
            self.log.error("(SP.disconnectDomains) Failed to get domains list")
            misc.logException(e)
            return False

        for dom in domDict:
            self.stopRepoStats(dom)
            try:
                SDF.produce(dom).disconnect()
            except Exception, e:
                misc.logException(e)
                # We should not rebuild non-active domains, because
                # they are probably disconnected from the host
                continue
        return True


    def disconnect(self, id, scsiKey, remove=False):
        """Disconnect a Host from specific storage pool.
        """
        self.log.info("(SP.disconnect) Disconnect from the storage pool %s",
            self.spUUID)

        if self.state == spDisconnected:
            return True
        if self.state == spDisconnecting:
            raise se.OperationInProgress("pool disconnect %s" % self.spUUID)
        self.state = spDisconnecting
        try:
            self.id = None
            self.scsiKey = None
            if os.path.exists(self._poolFile):
                os.unlink(self._poolFile)

            if self.hsmMailer:
                self.hsmMailer.stop()
                self.hsmMailer = None

            # Remove all links
            if os.path.exists(self.poolPath):
                misc.cleanupdir(self.poolPath)

            self.disconnectDomains()
        except:
            self.state = spError
            raise

        self.state == spDisconnected
        return True


    def reconnect(self):
        self.log.info("Trying to reconnect to pool: %s" % self.spUUID)
        try:
            file = open(self._poolFile, "r")
            for line in file:
                pair = line.strip().split("=")
                if len(pair) == 2:
                    if pair[0] == "id":
                        hostId = pair[1]
                    elif pair[0] == "scsiKey":
                        scsiKey = pair[1]
                    elif pair[0] == "sdUUID":
                        msdUUID = pair[1]
                    elif pair[0] == "version":
                        masterVersion = pair[1]
            file.close()
            if not (hostId and scsiKey and msdUUID and masterVersion):
                os.unlink(self._poolFile)
                return
            return self.connect(hostId, scsiKey, msdUUID, masterVersion)
        except:
            misc.logException("RECONNECT: Failed: %s" % (self.spUUID))
            os.unlink(self._poolFile)
            raise


    def createMaster(self, poolName, sdUUID, masterVersion, safeLease):
        """
        Create a fresh master file system directory tree
        """
        # THIS METHOD MUST BE RUN UNDER DOMAIN STORAGE LOCK
        self.log.info("(sp.createMaster) sdUUID=%s", sdUUID)
        dom = SDF.produce(sdUUID)
        dom.mountMaster()
        dom.createMasterTree()
        dom.unmountMaster()
        dom.setMetaParam(sd.ROLE, sd.MASTER_DOMAIN)
        dom.setMetaParam(sd.SPM_ID, "-1")
        dom.setMetaParam(sd.LVER, '-1')
        dom.setMetaParam(sd.MASTER_VER, masterVersion)
        dom.setMetaParam(sd.POOL, self.spUUID)
        dom.setMetaParam(sd.LOCK_POLICY, safeLease[sd.LOCK_POLICY])
        dom.setMetaParam(sd.LOCK_RENEWAL_INTERVAL_SEC, safeLease[sd.LOCK_RENEWAL_INTERVAL_SEC])
        dom.setMetaParam(sd.LEASE_TIME_SEC, safeLease[sd.LEASE_TIME_SEC])
        dom.setMetaParam(sd.IO_OP_TIMEOUT_SEC, safeLease[sd.IO_OP_TIMEOUT_SEC])
        dom.setMetaParam(sd.LEASE_RETRIES, safeLease[sd.LEASE_RETRIES])
        dom.setMetaParam(sd.POOL_DESCRIPTION, poolName)
        dom.setMetaParam(sd.DOMAINS, '%s:%s' % (sdUUID, sd.DOM_ATTACHED_STATUS))


    def reconstructMaster(self, poolName, msdUUID, domDict, masterVersion, safeLease):
        self.log.info("(StoragePool.reconstructMaster) spUUID=%s poolName=%s"
                      " master_sd=%s domDict=%s masterVersion=%s "
                      "leaseparams=(%s)",
                      self.spUUID, poolName, msdUUID, domDict, masterVersion,
                      str(safeLease))

        if msdUUID not in domDict:
            raise se.InvalidParameterException("masterDomain", msdUUID)

        try:
            # Seeing as we are just creating the pool then the host doesn't
            # have an assigned Id for this pool.  When locking the domain we must use an Id
            self.id = "1"
            # Master domain is unattached and all changes to unattached domains
            # must be performed under storage lock
            self.acquireDomainLock(msdUUID, safeLease)
        except:
            self.id = None
            raise
        try:
            self.createMaster(poolName, msdUUID, masterVersion, safeLease)
            self.refresh(msdUUID=msdUUID,masterVersion=masterVersion)

            # TBD: Run full attachSD?
            for sdUUID in domDict:
                # Add domain to domain list in pool metadata
                self.setDomains(sdUUID, domDict[sdUUID], remove=False, refresh=False)
        finally:
            self.releaseDomainLock(msdUUID)
            self.id = None


    def copyPoolMD(self, prevMd, newMD, masterVersion):
        domains = prevMd.getMetaParam(sd.DOMAINS)
        pool_descr = prevMd.getMetaParam(sd.POOL_DESCRIPTION)
        lver = prevMd.getMetaParam(sd.LVER)
        spmId = prevMd.getMetaParam(sd.SPM_ID)
        lockRenewalIntervalSec = prevMd.getMetaParam(sd.LOCK_RENEWAL_INTERVAL_SEC)
        leaseTimeSec = prevMd.getMetaParam(sd.LEASE_TIME_SEC)
        ioOpTimeoutSec = prevMd.getMetaParam(sd.IO_OP_TIMEOUT_SEC)
        leaseRetries = prevMd.getMetaParam(sd.LEASE_RETRIES)

        # Now insert pool metadata into new mastersd metadata
        newMD.setMetaParam(sd.DOMAINS, domains)
        newMD.setMetaParam(sd.POOL_DESCRIPTION, pool_descr)
        newMD.setMetaParam(sd.LVER, lver)
        newMD.setMetaParam(sd.SPM_ID, spmId)
        newMD.setMetaParam(sd.MASTER_VER, masterVersion)
        newMD.setMetaParam(sd.LOCK_RENEWAL_INTERVAL_SEC, lockRenewalIntervalSec)
        newMD.setMetaParam(sd.LEASE_TIME_SEC, leaseTimeSec)
        newMD.setMetaParam(sd.IO_OP_TIMEOUT_SEC, ioOpTimeoutSec)
        newMD.setMetaParam(sd.LEASE_RETRIES, leaseRetries)

    def __clearPoolMD(self, prevMd):
        # Clear metadata on prev master domain
        prevMd.setMetaParam(sd.SPM_ID, "-1")
        prevMd.setMetaParam(sd.LVER, '-1')
        # Finally remove master ROLE from old domain
        prevMd.setMetaParam(sd.ROLE, sd.REGULAR_DOMAIN)
        prevMd.setMetaParam(sd.MASTER_VER, 0)

    def __masterMigrate(self, sdUUID, msdUUID, masterVersion):
        curmsd = SDF.produce(sdUUID)
        newmsd = SDF.produce(msdUUID)

        # new 'master' should be in 'active' status
        domList = self.getDomains()
        if msdUUID not in domList:
            raise se.StorageDomainNotInPool(self.spUUID, msdUUID)
        if domList[msdUUID] != sd.DOM_ACTIVE_STATUS:
            raise se.StorageDomainNotActive(msdUUID)
        if newmsd.isISO():
            raise se.IsoCannotBeMasterDomain(msdUUID)
        if newmsd.isBackup():
            raise se.BackupCannotBeMasterDomain(msdUUID)

        # Copy master file system content to the new master
        src = os.path.join(curmsd.domaindir, sd.MASTER_DIR_NAME)
        dst = os.path.join(newmsd.domaindir, sd.MASTER_DIR_NAME)

        # Mount new master file system
        newmsd.mountMaster()
        # Make sure there is no cruft left over
        for dir in [os.path.join(dst, sd.VMS_DIR_NAME), os.path.join(dst, sd.TASKS_DIR_NAME)]:
            misc.cleanupdir(dir)

        cmd = ["%s cf - --exclude=lost+found -C %s . | %s xf - -C %s" % (constants.EXT_TAR, src, constants.EXT_TAR, dst)]
        rc = misc.execCmd(cmd, sudo=False, shell=True)[0]

        if rc:
            # Failed to copy the master data
            try:
                newmsd.unmountMaster()
            except Exception, e:
                misc.logException(e)
            raise se.StorageDomainMasterCopyError(msdUUID)

        self.copyPoolMD(curmsd, newmsd, masterVersion)

        path = newmsd.getMDPath()
        if not path:
            newmsd.unmountMaster()
            raise se.StorageDomainLayoutError("domain", msdUUID)

        # Acquire safelease lock on new master
        try:
            # Reset SPM lock because of the host still SPM
            # It will speedup new lock acquiring
            newmsd.initSPMlease()
            self.acquireDomainLock(msdUUID)
        except Exception, e:
            misc.logException(e)
            self.releaseDomainLock(msdUUID)
            newmsd.unmountMaster()
            raise
        self.log.debug("masterMigrate - lease acquired successfully")

        try:
            # Now mark new domain as 'master'
            # if things break down here move the master back pronto
            newmsd.setMetaParam(sd.ROLE, sd.MASTER_DOMAIN)
        except Exception, e:
            misc.logException(e)
            self.releaseDomainLock(msdUUID)
            newmsd.unmountMaster()
            raise

        # From this point on we have a new master and should not fail
        try:
            # Now recreate 'mastersd' link
            # we can use refresh() to do the job
            self.refresh(msdUUID, masterVersion)

            # From this point on there is a new master domain in the pool
            # Now that we are beyond the criticial point we can clean up things
            self.__clearPoolMD(curmsd)

            # Clean up the old data from previous master fs
            for dir in [os.path.join(src, sd.VMS_DIR_NAME), os.path.join(src, sd.TASKS_DIR_NAME)]:
                misc.cleanupdir(dir)

            # NOTE: here we unmount the *previous* master file system !!!
            curmsd.unmountMaster()
        except Exception, e:
            misc.logException(e)

        try:
            # Release old lease
            self.releaseDomainLock(sdUUID)
        except Exception, e:
            misc.logException(e)


    def __unmountLastMaster(self, sdUUID):
        curmsd = SDF.produce(sdUUID)
        # Check if it's last domain and allow it detaching
        dl = self.getDomains(activeOnly=True)
        domList = dl.keys()
        if curmsd.sdUUID in domList:
            domList.remove(curmsd.sdUUID)
        for item in domList:
            domain = SDF.produce(item)
            if domain.isData():
                # Failure, we have at least one more data domain
                # in the pool and one which can become 'master'
                raise se.StoragePoolHasPotentialMaster(item)
        curmsd.unmountMaster()
        self.masterDomain = None

    def masterMigrate(self, sdUUID, msdUUID, masterVersion):
        self.log.info("(StoragePool.masterMigrate) sdUUID=%s spUUID=%s msdUUID=%s",
           sdUUID,  self.spUUID, msdUUID)

        # Check if we are migrating to or just unmounting last master
        if msdUUID != sd.BLANK_UUID:
            self.__masterMigrate(sdUUID, msdUUID, masterVersion)
            return False    # not last master

        self.__unmountLastMaster(sdUUID)
        return True     # last master


    def getDomainPools(self, dom):
        poolList = []
        pl = dom.getMetaParam(key=sd.POOL)
        if pl:
            poolList = pl.split(",")
        return poolList


    def _attachSD(self, sdUUID):
        msdType = sd.name2type(self.getMasterDomain().getInfo()["type"])
        if msdType in (sd.ISCSI_DOMAIN, sd.FCP_DOMAIN) and len(self.getDomains()) >= MAX_DOMAINS:
            raise se.TooManyDomainsInStoragePoolError()

        dom = SDF.produce(sdUUID)
        poolList = self.getDomainPools(dom=dom)
        # Avoid attach domains if owned by another pool (except ISO domain)
        if not dom.isISO():
            if len(poolList):
                raise se.StorageDomainAlreadyAttached(poolList[0], sdUUID)
        if self.spUUID in poolList:
            misc.logException("sp._attachSD: Can't add pool %s to the pools list %s, it already exists" % (self.spUUID, str(poolList)))
            # TODO - raise and catch above
            return

        poolList.append(self.spUUID)
        dom.setMetaParam(sd.POOL, ",".join(poolList))


    def attachSD(self, sdUUID):
        """Attach a storage domain to the storage pool.
           This marks the storage domain as "attached" and links it
           to the storage pool
            'sdUUID' - storage domain UUID
        """
        self.log.info("(StoragePool.attachSD) sdUUID=%s spUUID=%s", sdUUID, self.spUUID)
        self.acquireDomainLock(sdUUID)
        try:
            # Refresh damain's cache
            SDF.produce(sdUUID).getMetaParam(key=sd.POOL, cache=False)
            domList = self.getDomains()
            if sdUUID in domList and domList[sdUUID] != sd.DOM_ATTACHED_STATUS:
                raise se.StorageDomainStateTransitionIllegal(sdUUID, domList[sdUUID], sd.DOM_ATTACHED_STATUS)
            # Mark domain as pool member
            self._attachSD(sdUUID)
            # Add domain to domain list in pool metadata
            self.setDomains(sdUUID, sd.DOM_ATTACHED_STATUS, remove=False)
            # Finally update pool tree
            return True
        finally:
            self.releaseDomainLock(sdUUID)


    def forcedDetachSD(self, sdUUID):
        # Remove domain from domain list in pool metadata
        self.setDomains(sdUUID, sd.DOM_UNATTACHED_STATUS, remove=True, refresh=False)
        # Finally update pool tree
        self.refresh()


    def _detachSD(self, sdUUID):
        """Detach a storage domain from a storage pool.
           This removes the storage domain entry in the storage pool meta-data
           and leaves the storage domain in 'unattached' status.
            'spUUID' - storage pool UUID
        """
        dom = SDF.produce(sdUUID)
        # Remove domain from the pool
        poolList = self.getDomainPools(dom=dom)
        if self.spUUID not in poolList:
            misc.logException("sp._detachSD: Can't remove pool %s from domain %s pool list %s, it does not exist" % (self.spUUID, sdUUID, str(poolList)))
            # TODO - raise and catch above
            return
        poolList.remove(self.spUUID)
        # Make sure that ROLE is not MASTER_DOMAIN (just in case)
        dom.setMetaParam(sd.ROLE, sd.REGULAR_DOMAIN)
        # Last thing to do is to remove pool from domain
        dom.setMetaParam(sd.POOL, ",".join(poolList))
        # do any required cleanup
        dom.disconnect()


    def detachSD(self, sdUUID, msdUUID, masterVersion):
        """Detach a storage domain from a storage pool.
           This removes the storage domain entry in the storage pool meta-data
           and leaves the storage domain in 'unattached' status.
            'sdUUID' - storage domain UUID
            'msdUUID' - master storage domain UUID
            'masterVersion' - new master storage domain version
        """
        self.log.info("(StoragePool.detachSD) sdUUID=%s spUUID=%s msdUUID=%s",
           sdUUID,  self.spUUID, msdUUID)

        e = None

        domClass = SDF.produce(sdUUID).getMetaParam(key=sd.CLASS, cache=True)
        if type == "Iso":
            self.acquireDomainLock(sdUUID)
        try:
            # Refresh damain's cache
            SDF.produce(sdUUID).getMetaParam(key=sd.POOL, cache=False)
            try:
                # Avoid detach domains if not owned by pool
                self.validateAttachedDomain(sdUUID)
                domList = self.getDomains()
                sd.validateSDStateTransition(sdUUID, domList[sdUUID], sd.DOM_UNATTACHED_STATUS)

                lastDom = False
                # If the domain being detached is the 'master', move all pool
                # metadata to the new 'master' domain (msdUUID)
                if sdUUID == self.getMasterDomain().sdUUID:
                    lastDom = self.masterMigrate(sdUUID, msdUUID, masterVersion)

                # Remove domain from domain list in pool metadata
                self.setDomains(sdUUID, sd.DOM_UNATTACHED_STATUS, remove=True, refresh=False)
                # Remove pool info from domain metadata
                self._detachSD(sdUUID)
            except Exception, e:
                misc.logException(e)
        finally:
            if type == "Iso":
                self.releaseDomainLock(sdUUID)

        # Finally update pool tree
        try:
            self.refresh()
        except se.StoragePoolMasterNotFound, err:
            if e:
                raise e
            # When last domain (that was a 'master') detached
            # refresh() will raise exception StoragePoolMasterNotFound.
            # It's the correct behaviour
            if not lastDom:
                raise se.StorageDomainDetachError(str(err))
        if e:
            raise e


    def activateSD(self, sdUUID):
        """Activate a storage domain that is already a member in a storage pool.
           Validate that the storage domain is owned by the storage pool.
            'sdUUID' - storage domain UUID
        """
        self.log.info("(StoragePool.activateSD) sdUUID=%s spUUID=%s", sdUUID,  self.spUUID)

        # Avoid domain activation if not owned by pool
        self.validateAttachedDomain(sdUUID)
        domList = self.getDomains()
        dom = SDF.produce(sdUUID)
        sd.validateSDStateTransition(sdUUID, domList[sdUUID], sd.DOM_ACTIVE_STATUS)

        # Do nothing if already active
        if domList[sdUUID] == sd.DOM_ACTIVE_STATUS:
            return True

        dom.activate()
        # set domains also do rebuild
        self.setDomains(sdUUID, sd.DOM_ACTIVE_STATUS, remove=False)
        return True


    def deactivateSD(self, sdUUID, msdUUID, masterVersion):
        """
        Deactivate a storage domain.
        Validate that the storage domain is owned by the storage pool.
        Change storage domain status to "Attached" in the storage pool meta-data.

        :param sdUUID: The UUID of the storage domain you want to deactivate.
        :param msdUUID: The UUID of the pool that owns this domain.
                        If the deactivated domain *is* the master domain put a blank UUID.
        :param masterVersion: new master storage domain version
        """
        self.log.info("(StoragePool.deactivateSD) sdUUID=%s spUUID=%s msdUUID=%s",
           sdUUID,  self.spUUID, msdUUID)
        #Maybe there should be information in the exception that the UUID is
        #not invalid because of its format but because it is equal to the SD. Will be less confusing.
        if sdUUID == msdUUID:
            raise se.InvalidParameterException("msdUUID", msdUUID)

        # Check if deactivating master domain
        deactivatingMaster = (sdUUID == self.getMasterDomain().sdUUID)

        if msdUUID != BLANK_POOL_UUID:
            # If we got a new master domain it implies we are deactivating the master domain
            if not deactivatingMaster:
                raise se.InvalidParameterException("msdUUID not blank", msdUUID)
            # when deactivating master domain the masterVersion must be higher
            self.validatePoolMVerHigher(masterVersion)


        domList = self.getDomains()
        if sdUUID not in domList:
            raise se.StorageDomainNotInPool(self.spUUID, sdUUID)

        sd.validateSDStateTransition(sdUUID, domList[sdUUID], sd.DOM_ATTACHED_STATUS)

        # Allow deactivate even if the domain is faulty
        try:
            dom = SDF.produce(sdUUID)
            dom.validate()
            pools = self.getDomainPools(dom=dom)
            if pools and self.spUUID not in pools:
                raise se.StorageDomainNotInPool(self.spUUID, sdUUID)
        except Exception, e:
            misc.logException(e)
            if not deactivatingMaster:
                self.setDomains(sdUUID, sd.DOM_ATTACHED_STATUS, remove=False)
                # It is not an error to deactivate a faulty domain, therefore no need to throw exception
                return
            # One cannot deactivate a faulty master domain.
            raise

        if deactivatingMaster:
            self.masterMigrate(sdUUID, msdUUID, masterVersion)
        dom.deactivate()
        self.setDomains(sdUUID, sd.DOM_ATTACHED_STATUS, remove=False)


    def refreshDomain(self, d, msdUUID, linkName):
        d.refresh()
        dom = d.sdUUID

        self.log.info("(SP.__rebuild) Linking SD %s to %s", dom, d.domaindir)
        # Rebuild link if it does not exist or is wrong
        if os.path.lexists(linkName):
            self.log.debug("check path %s " % (os.stat(linkName)))
            curr = None
            try:
                 curr = os.readlink(linkName)
            except:
                self.log.warning("(SP.__rebuild): failed to read link %s" % linkName)
            if curr != d.domaindir:
                os.remove(linkName)
                os.symlink(d.domaindir, linkName)
        else:
            os.symlink(d.domaindir, linkName)
        # In case of master domain make a special 'mastersd' link
        if dom == msdUUID:
            masterlink = os.path.join(self.poolPath, POOL_MASTER_DOMAIN)
            # Rebuild link if it does not exist or is wrong
            if os.path.lexists(masterlink):
                if os.readlink(masterlink) != d.sdUUID:
                    os.remove(masterlink)
                    os.symlink(d.sdUUID, masterlink)
            else:
                os.symlink(d.sdUUID, masterlink)
        # Finally, if non-master domain is marked as master (can happen after master reconstruct)
        elif d.isMaster():
            d.setMetaParam(sd.ROLE, sd.REGULAR_DOMAIN)
            d.setMetaParam(sd.MASTER_VER, 0)

    def __rebuild(self, msdUUID=None, masterVersion=None):
        """
        Rebuild storage pool.
        """
        # master domain must be refreshed first
        msdUUID = self.getMasterDomain(msdUUID=msdUUID, masterVersion=masterVersion).sdUUID

        if not os.path.exists(self.poolPath):
            os.mkdir(self.poolPath)

        # Find out all domains for future cleanup
        domainpat = os.path.join(self.poolPath, UUID_PATTERN)
        cleanupdomains = glob(domainpat)

        # We should not rebuild non-active domains, because
        # they are probably disconnected from the host
        domList = self.getDomains(activeOnly=True).keys()

        # Always try to build master links
        if msdUUID not in domList:
            domList.append(msdUUID)

        for dom in domList:
            try:
                d = SDF.produce(dom)
            except se.StorageDomainDoesNotExist, e:
                # We should not rebuild a non-master active domain
                # if it is disconnected. Log the error and continue
                misc.logException(e)
                continue

            try:
                linkName = os.path.join(self.poolPath, dom)
                self.refreshDomain(d, msdUUID, linkName)
                # Remove domain from potential cleanup
                if linkName in cleanupdomains:
                    cleanupdomains.remove(linkName)
            except Exception, e:
                misc.logException(e)
                continue

        # Clenup old trash from the pool
        for i in cleanupdomains:
            try:
                os.remove(i)
            except OSError, e:
                if e.errno != 2:
                    misc.logException(e)
            except Exception, e:
                misc.logException(e)


    def refresh(self, msdUUID=None, masterVersion=None):
        """Refresh storage pool.
            'msdUUID' - master storage domain UUID
        """
        # Make sure the StorageDomainFactory has its internal cache refreshed
        SDF.refresh()
        self.__rebuild(msdUUID=msdUUID, masterVersion=masterVersion)


    def updateVM(self, vmList, sdUUID=None):
        """Update VMs.
            'vmList' - [{'vm':vmUUID,'ovf','imglist':'imgUUID1,imgUUID2,...'},...]
            'sdUUID' - target domain UUID, if not None, VM Images and the master tree
                       must be located on this domain.
                       If sdUUID is None, the update is on the pool, and therefore the
                       master domain will be updated.
        """
        self.log.info("(StoragePool.updateVM) spUUID=%s sdUUID=%s",
                      self.spUUID, sdUUID)

        vms = self._getVMsPath(sdUUID)
        # We should exclude 'masterd' link from IMG_METAPATTERN globing
        vmUUID = ovf = imgList = ''
        for vm in vmList:
            if not vm:
                continue
            try:
                vmUUID = vm['vm']
                ovf = vm['ovf']
                imgList = vm['imglist'].split(',')
                self.log.info("(StoragePool.updateVM) vmUUID=%s imgList=%s",
                              vmUUID, str(imgList))
            except KeyError:
                raise se.InvalidParameterException("vmList", str(vmList))

            vmPath = os.path.join(vms, vmUUID)
            if misc.fileexists(vmPath):
                misc.cleanupdir(vmPath, ignoreErrors = False)
            os.mkdir(vmPath)
            # TBD: BZ#556742 - getImageDomainsList throws exception when ANY
            # data domain is not available.  Seeing as the symbolic link created
            # below is not functional, for now the entire section is marked off.
            # consider deleting!
            #for imgUUID in imgList:
            #    if not imgUUID:
            #        continue
            #    if not sdUUID or sdUUID == sd.BLANK_UUID:
            #        domlist = self.getImageDomainsList(imgUUID)
            #        if not domlist:
            #            raise se.ImageMissingFromVm(imgUUID, vmUUID)
            #        dom = domlist[0]
            #    else:
            #        imageslist = SDF.produce(sdUUID).getAllImages()
            #        if not imgUUID in imageslist:
            #            raise se.ImageNotOnTargetDomain(imgUUID, vmUUID, sdUUID)
            #        dom = sdUUID

            #    os.symlink(os.path.join(self.poolPath, dom, sd.DOMAIN_IMAGES, imgUUID),
            #        os.path.join(vmPath, imgUUID))
            open(os.path.join(vmPath, vmUUID+'.ovf'),'w').write(ovf)


    def removeVM(self, vmList, sdUUID=None):
        """Remove VMs.
            'vmList' - vmUUID1,vmUUID2,...
        """
        self.log.info("(StoragePool.removeVM) spUUID=%s vmList=%s sdUUID=%s",
            self.spUUID, str(vmList), sdUUID)
        vms = self._getVMsPath(sdUUID)
        vmUUIDs = vmList.split(',')
        for vm in vmUUIDs:
            if os.path.exists(os.path.join(vms, vm)):
                misc.cleanupdir(os.path.join(vms, vm))


    def setDescription(self, descr):
        """Set storage pool description.
            'descr' - pool description
        """
        if len(descr) > MAX_POOL_DESCRIPTION_SIZE:
            raise se.StoragePoolDescriptionTooLongError()

        self.log.info("(StoragePool.setDescription) spUUID=%s descr=%s",
            self.spUUID, descr)
        self.setMetaParam(sd.POOL_DESCRIPTION, descr)


    def extendVolume(self, sdUUID, volumeUUID, size, isShuttingDown=None):
        SDF.produce(sdUUID).extendVolume(volumeUUID, size,
            isShuttingDown)

    def getInfo(self):
        """
        Get storage pool info.
        """
        ##self.log.info("(StoragePool.getInfo) Get info of the storage pool %s",
        ##    self.spUUID)
        if not self.spUUID:
            raise se.StoragePoolInternalError

        info = {'type': '', 'name': '', 'domains': '', 'master_uuid': '', 'master_ver': 0,
            'lver': -1, 'spm_id': -1, 'isoprefix': '', 'pool_status': spUninit}
        list_and_stats = {}

        try:
            metadata = self.getMasterDomain().getMetadata()
        except:
            misc.logException("Couldn't read from master domain")
            raise se.StoragePoolMasterNotFound(self.spUUID)

        try:
            info['type'] = metadata[sd.TYPE]
            info['domains'] = metadata[sd.DOMAINS]
            try:
                info['name'] = metadata[sd.POOL_DESCRIPTION]
            except KeyError:
                info['name'] = ''
                self.log.warning("Missing pool description in metadata",
                                 exc_info=True)
            info['lver'] = metadata[sd.LVER]
            info['spm_id'] = metadata[sd.SPM_ID]
            info['master_uuid'] = metadata[sd.SDUUID]
            info['master_ver'] = metadata[sd.MASTER_VER]
        except:
            self.log.error("Pool metadata error", exc_info=True)
            raise se.StoragePoolActionError(self.spUUID)


        # Get info of all pool's domains
        domDict = self.getDomains()
        repoStats = self.getRepoStats()
        for item in domDict:
            # Return statistics for active domains only
            stats = {}
            alerts = []
            if domDict[item] == sd.DOM_ACTIVE_STATUS:
                try:
                    dom = SDF.produce(item)
                    if dom.isISO():
                        info['isoprefix'] = os.path.join(self.poolPath, item,
                                              sd.DOMAIN_IMAGES, sd.ISO_IMAGE_UUID)
                    if item in repoStats:
                        stats['disktotal'] = repoStats[item]['disktotal']
                        stats['diskfree'] = repoStats[item]['diskfree']
                        if not repoStats[item]['mdavalid']:
                            alerts.append({'code':se.SmallVgMetadata.code,
                                           'message':se.SmallVgMetadata.message})
                            self.log.warning("VG %s's metadata size too small",
                                 dom.sdUUID)

                        if not repoStats[item]['mdathreshold']:
                            alerts.append({'code':se.VgMetadataCriticallyFull.code,
                                           'message':se.VgMetadataCriticallyFull.message})
                            self.log.warning("VG %s's metadata size exceeded critical size",
                                 dom.sdUUID)
                        stats['alerts'] = alerts
                except se.StorageException, e:
                    # No error is returned when an erroneous domain is found
                    misc.logException(e)
                    stats = {}
            stats['status'] = domDict[item]
            list_and_stats[item] = stats

        info["pool_status"] = self.state
        return dict(info=info, dominfo=list_and_stats)


    def getIsoDomain(self):
        """Get pool's ISO domain if active
        """
        domDict = self.getDomains(activeOnly=True)
        for item in domDict:
                dom = SDF.produce(item)
                if dom.isISO():
                    return dom
        return None

    def getIsoList(self, spUUID, extension):
        """Get list of all ISO/Floppy volumes
            'spUUID' - storage pool UUID
        """
        self.log.info("(SP.getIsoList) spUUID=%s, extension=%s", spUUID, extension)
        isoDom = self.getIsoDomain()
        if not isoDom:
            return []
        isolist = isoDom.getIsoList(extension)
        self.log.info("(SP.getIsoList) List of ISO/Floppy images is: %s", str(isolist))
        return isolist

    def setMetaParam(self, key, value):
        """ Set key:value in pool metadata file
        """
        if not self.spUUID:
            raise se.StoragePoolInternalError
        master = self.getMasterDomain()
        master.setMetaParam(key=key, value=value)

    def getMetaParam(self, key, cache=True):
        """ Get parameter from pool metadata file
        """
        if not self.spUUID:
            raise se.StoragePoolInternalError
        master = self.getMasterDomain()
        try:
            return master.getMetaParam(key=key, cache=cache)
        except:
            misc.logException("Couldn't read from master domain")
            raise se.StoragePoolMasterNotFound(self.spUUID)

    def getMasterDomain(self, msdUUID=None, masterVersion=None):
        if msdUUID or not self.masterDomain:
            self.masterDomain = self.findMasterDomain(msdUUID=msdUUID, masterVersion=masterVersion)
            self.refreshDomains()
        return self.masterDomain

    def findMasterDomain(self, msdUUID=None, masterVersion=-1):
        """ Get master domain of this pool
            'spUUID' - storage pool UUID
        """
        masterdomain = None
        e = None

        # Validate params, if given
        try:
            # Make sure we did not receive a version without a domain
            if not msdUUID:
                masterVersion = -1
            # Make sure that version is an integer
            masterVersion = int(masterVersion)
        except:
            msdUUID = None
            masterVersion = -1

        # Find master domain with highest version
        dl = SDF.getAllDomains()
        for d in dl:
            try:
                pools = self.getDomainPools(d)
                if (self.spUUID in pools):
                    if not d.isMaster():
                        if msdUUID != d.sdUUID:
                            continue
                        self.log.error("(StoragePool.findMasterDomain) Domain  %s is not master", d.sdUUID)
                        e = se.StoragePoolWrongMaster(self.spUUID, d.sdUUID)
                        break

                    # get domain's master version
                    ver = d.getMasterVer()
                    # For backwards compatibility we must support not receiving msdUUID.
                    # When we do receive it, make sure we can't find another domain with
                    # same or higher version (could happen for example if during
                    # reconstructMaster the network link to the vdc went down)
                    if ver == masterVersion:
                        if not msdUUID or msdUUID != d.sdUUID:
                            self.log.error("(StoragePool.findMasterDomain) Found another master domain %s with same master"\
                                           " version than input: ver=%s(%s)", d.sdUUID, ver, masterVersion)
                            e = se.StoragePoolTooManyMasters(self.spUUID)
                            break
                        masterdomain = d
                        e = None

                    elif ver > masterVersion:
                        masterdomain = d
                        masterVersion = ver
                        if msdUUID:
                            e = se.StoragePoolWrongMaster(self.spUUID, d.sdUUID)
                            self.log.error("(StoragePool.findMasterDomain) Found master domain %s with higher master"\
                                           " version than input: ver=%s(%s)", d.sdUUID, ver, masterVersion)
                            break

                    else: #if ver < masterVersion:
                        if msdUUID == d.sdUUID:
                            masterdomain = d
                            e = se.StoragePoolWrongMaster(self.spUUID, d.sdUUID)
                            self.log.error("(StoragePool.findMasterDomain) Found master domain %s with lower master"\
                                           " version than input: ver=%s(%s)", d.sdUUID, ver, masterVersion)
            except:
                # Keep going even if we are barfing on some domains
                misc.logException("Couldn't determine domain status (%s)" % d.sdUUID)

        if e:
            raise e

        if not masterdomain:
            raise se.StoragePoolMasterNotFound(self.spUUID)

        return masterdomain


    def setDomains(self, sdUUID, status, remove=False, refresh=True):
        domDict = {}
        domDict = self.getDomains()
        domDict[sdUUID] = status.capitalize()
        if remove and sdUUID in domDict.keys():
            del domDict[sdUUID]
        domains = ','.join([ '%s:%s' % (k, v) for k, v in domDict.iteritems()])
        self.setMetaParam(sd.DOMAINS, domains)
        self.log.info("(StoragePool.setDomains) Set storage pool domains: %s",
            str(domains))
        self.refreshDomains()
        if refresh:
            self.refresh()

    @misc.samplingmethod
    def refreshDomains(self):
        # domain list it's list of sdUUID:status
        # sdUUID1:status1,sdUUID2:status2,...
        dom = {}
        domains = []
        domList = self.getMetaParam(sd.DOMAINS, cache=False)
        if domList:
            domains = domList.split(",")

        self.log.debug("SP.refreshDomains: sp %s old domains %s master dom list %s" % (self.spUUID, self.domains, domains))
        # Now convert the list of "sdUUID:Status" strings
        # into dictionary.
        for i in domains:
            k, v = i.split(':')
            dom[k.strip("'")] = v.strip("'").capitalize()

        # Stop RepoStats threads of all the now "Inactive" or absent domains
        for d in self.domains:
            try:
                if (self.domains[d] == sd.DOM_ACTIVE_STATUS and
                    (d not in dom or dom[d] != sd.DOM_ACTIVE_STATUS)):
                    self.log.debug("SP.refreshDomains: sp %s remove domain %s" % (self.spUUID, d))
                    self.stopRepoStats(d)
            except se.StorageException, e:
                misc.logException(e)

        # Start RepoStats threads of all the newly activated domains
        for d in dom:
            # FIXME:  RepoStats should be moving out from the domain object
            try:
                if (dom[d] == sd.DOM_ACTIVE_STATUS and
                    (d not in self.domains or
                     self.domains[d] != sd.DOM_ACTIVE_STATUS)):
                    self.log.debug("SP.refreshDomains: sp %s add domain %s" % (self.spUUID, d))
                    self.startRepoStats(d)
            except se.StorageException, e:
                misc.logException(e)

        self.log.debug("SP.refreshDomains: sp %s new domains %s" % (self.spUUID, dom))
        self.domains = dom


    def getDomains(self, activeOnly=False):
        # domain list it's list of sdUUID:status
        # sdUUID1:status1,sdUUID2:status2,...
        dom = {}
        for i in self.domains:
            if not activeOnly or self.domains[i] == sd.DOM_ACTIVE_STATUS:
                dom[i] = self.domains[i]

        self.log.info("(StoragePool.getDomains) Get storage pool domains: %s",
            str(dom))
        # return dictionary {sdUUID1:status1,sdUUID2:status2}
        return dom

    def checkBackupDomain(self):
        domDict = self.getDomains(activeOnly=True)
        for sdUUID in domDict:
            dom = SDF.produce(sdUUID)
            if dom.isBackup():
                dom.mountMaster()
                # Master tree should be exist in this point
                # Recreate it if not.
                dom.createMasterTree()


    def getImageDomainsList(self, imgUUID, datadomains=True):
        """Get list of all domains in the pool that contain imgUUID
            'imgUUID' - image UUID
        """
        # TODO: get rid of this verb and let management query each domain separately
        #  the problem with current implementation is that when a domain is not accesible
        #  the error must be ignored and management can reach wrong conclusions.
        domainsdict = self.getDomains(activeOnly=True)
        domainslist = []

        for dom in domainsdict:
            try:
                d = SDF.produce(dom)
            except Exception, e:
                # Pass over invisible active domains
                misc.logException(e)
                continue

            if datadomains and not d.isData():
                continue
            imageslist = d.getAllImages()
            if imgUUID in imageslist:
                domainslist.append(dom)

        return domainslist

    def isMember(self, sdUUID, checkActive=False):
        """ Check if domain is memeber in the pool.
        """
        return sdUUID in self.getDomains(activeOnly=checkActive)

    def _getVMsPath(self, sdUUID):
        """ Return general path of VMs from the pool.
            If 'sdUUID' is given then return VMs dir within it.
        """
        if sdUUID and sdUUID != sd.BLANK_UUID:
            domDict = self.getDomains()
            if domDict[sdUUID] != sd.DOM_ACTIVE_STATUS:
                raise se.StorageDomainNotActive(sdUUID)
            vmpath = os.path.join(self.poolPath, sdUUID, sd.MASTER_DIR_NAME, sd.VMS_DIR_NAME)
        # Get VMs path from the pool (from the master domain)
        else:
            vmpath = os.path.join(self.poolPath, sd.VMS_DIR_NAME)

        if not os.path.exists(vmpath):
            raise se.VMPathNotExists(vmpath)
        return vmpath

    def getVmsList(self, sdUUID=None):
        """ Get list of VMs from the pool.
            If 'sdUUID' given get list of VMs from it, if it's Backup domain.
            'sdUUID' - UUID of Backup domain
        """
        self.log.info("(StoragePool.getVmsList) spUUID=%s sdUUID=%s",
            self.spUUID, sdUUID)

        vmsPath = self._getVMsPath(sdUUID)
        # find out VMs list
        VM_PATTERN = os.path.join(vmsPath, UUID_PATTERN)
        vms = glob(VM_PATTERN)
        vmList = [os.path.basename(i) for i in vms]
        self.log.info("(StoragePool.getVmsList) vmList=%s", str(vmList))

        return vmList

    def getVmsInfo(self, sdUUID=None, vmList=None):
        """ Get list of VMs with their info from the pool.
            If 'sdUUID' are given get list of VMs from it, if it's Backup domain
            If 'vmList' are given get info of these VMs only
        """
        self.log.info("(StoragePool.getVmsInfo) spUUID=%s sdUUID=%s vmList=%s",
            self.spUUID, sdUUID, str(vmList))

        vmsInfo = {}
        vmsPath = self._getVMsPath(sdUUID)

        # Find out relevant VMs
        if not vmList:
            vmList = self.getVmsList(sdUUID)

        self.log.info("(StoragePool.getVmsInfo) vmList=%s", str(vmList))

        for vm in vmList:
            vm_path = os.path.join(vmsPath, vm)
            # If VM doesn't exists, ignore it silently
            if not os.path.exists(vm_path):
                continue
            ovfPath = os.path.join(vm_path, vm+'.ovf')
            if not os.path.lexists(ovfPath):
                raise se.MissingOvfFileFromVM(vm)

            ovf = open(ovfPath).read()
            vmsInfo[vm] = ovf

        return vmsInfo

    def check(self):
        poolstatus = 0
        baddomains = {}
        message = "Pool OK"
        try:
            masterdomain = self.findMasterDomain()
            spmId = self.getMetaParam(sd.SPM_ID, cache=False)
            domains = self.getDomains(activeOnly=True)

            for dom in domains:
                d = SDF.produce(dom)
                domstatus = d.checkDomain(spUUID=self.spUUID)
                if domstatus["domainstatus"] != 0:
                    baddomains[dom] = domstatus
                    poolstatus = se.StoragePoolCheckError.code
                    message = "Pool has bad domains"
        except se.StorageException, e:
            poolstatus = e.code
            message = str(e)
        except:
            poolstatus = se.StorageException.code
            message = "Pool is bad"

        return dict(poolstatus = poolstatus, baddomains = baddomains,
                    masterdomain = masterdomain.sdUUID, spmhost=spmId,
                    message = message)


    def _repostats(self, sduuid):
        # self.selftest() should return True if things are looking good
        # and False otherwise
        stats = {}
        code = 0
        try:
            domain = SDF().produce(sduuid)
            if not domain.selftest():
                code = 200

            res = domain.getStats()
            stats.update(res)
            # Add here more selftests if needed
            # Fill stats to get it back to the caller
            # Keys 'finish' and 'result' are reserved and may not be used
            stats['masterValidate'] = domain.validateMaster()
        except se.StorageException, e:
            code = e.code

        return stats, code


    def startRepoStats(self, domain):
        statthread = self.repostats.get(domain)
        self.log.debug("%s stat %s", domain, statthread)
        if not statthread:
            statthread = StatsThread(self._repostats, domain)
            statthread.start()
            self.repostats[domain] = statthread


    def stopRepoStats(self, domain):
        statthread = self.repostats.pop(domain, None)
        self.log.debug("%s stat %s", domain, statthread)
        if statthread:
            statthread.stop()


    def getRepoStats(self):
        repostats = self.repostats.copy()
        result = {}
        for d in repostats:
            result[d] = repostats[d].getStatsResults()
        return result
